﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace TextEmbedding.Core
{
    public interface ITokenizer
    {
        public string[] Tokenize(string text, bool shouldCutAll);
    }
}
